{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d153f048-0fcf-4e66-b23e-94ee35cd6570",
   "metadata": {},
   "source": [
    "# 1. 加载模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "24d3dd45-f008-4306-ae19-1a6aed2fcd4d",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-12-23T10:03:05.769736Z",
     "iopub.status.busy": "2024-12-23T10:03:05.769558Z",
     "iopub.status.idle": "2024-12-23T10:03:08.630110Z",
     "shell.execute_reply": "2024-12-23T10:03:08.629535Z",
     "shell.execute_reply.started": "2024-12-23T10:03:05.769714Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from transformers import Qwen2VLForConditionalGeneration, AutoTokenizer, AutoProcessor\n",
    "from qwen_vl_utils import process_vision_info\n",
    "from modelscope import snapshot_download\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7c076aef-1703-47f0-9daa-f91be727aa85",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-12-23T10:03:08.631872Z",
     "iopub.status.busy": "2024-12-23T10:03:08.631284Z",
     "iopub.status.idle": "2024-12-23T10:03:08.634380Z",
     "shell.execute_reply": "2024-12-23T10:03:08.633786Z",
     "shell.execute_reply.started": "2024-12-23T10:03:08.631848Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 定义模型路径\n",
    "model_directory = \"/mnt/workspace/models/Qwen2-VL-2B-Instruct\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6ad6fc3a-e8a1-494f-a832-629885ca9962",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-12-23T10:03:08.635255Z",
     "iopub.status.busy": "2024-12-23T10:03:08.635046Z",
     "iopub.status.idle": "2024-12-23T10:03:11.839330Z",
     "shell.execute_reply": "2024-12-23T10:03:11.838696Z",
     "shell.execute_reply.started": "2024-12-23T10:03:08.635234Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "`Qwen2VLRotaryEmbedding` can now be fully parameterized by passing the model config through the `config` argument. All other arguments will be removed in v4.46\n",
      "Loading checkpoint shards: 100%|██████████| 2/2 [00:02<00:00,  1.01s/it]\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    # 加载预训练模型和处理器到可用设备（CPU或GPU）\n",
    "    model = Qwen2VLForConditionalGeneration.from_pretrained(\n",
    "        model_directory, torch_dtype=torch.float16, device_map=\"auto\"\n",
    "    )\n",
    "    processor = AutoProcessor.from_pretrained(model_directory)\n",
    "except Exception as e:\n",
    "    print(f\"加载模型时出错: {e}\")\n",
    "    raise"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "681bcff7-4879-474d-93ec-b5776b76da31",
   "metadata": {},
   "source": [
    "# 2. 遍历数据获取结果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "cfbb2b4c-d536-4b05-b3cc-e84036eaa707",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-12-23T10:23:26.483486Z",
     "iopub.status.busy": "2024-12-23T10:23:26.483091Z",
     "iopub.status.idle": "2024-12-23T10:23:26.491493Z",
     "shell.execute_reply": "2024-12-23T10:23:26.490822Z",
     "shell.execute_reply.started": "2024-12-23T10:23:26.483459Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def get_response(text, image_path):\n",
    "    sys_prompt = \"\"\"\n",
    "    请对用户输入的内容和图片进行分析！\n",
    "    将分析结果输出为JSON，key是output，value是分析结果，如：\n",
    "    {\n",
    "        \"output\": \"分析结果\"\n",
    "    }\n",
    "    \"\"\"\n",
    "    messages = [\n",
    "        {\"role\": \"system\", \"content\": sys_prompt},\n",
    "        {\n",
    "            \"role\": \"user\",\n",
    "            \"content\": [\n",
    "                {\n",
    "                    \"type\": \"image\",\n",
    "                    \"image\": image_path,\n",
    "                },\n",
    "                {\"type\": \"text\", \"text\": text},\n",
    "            ],\n",
    "        }\n",
    "    ]\n",
    "    try:\n",
    "        # 使用处理器准备文本模板用于推理\n",
    "        text_template = processor.apply_chat_template(\n",
    "            messages, tokenize=False, add_generation_prompt=True\n",
    "        )\n",
    "\n",
    "        # 处理视觉信息（图像或视频），准备输入给模型\n",
    "        image_inputs, video_inputs = process_vision_info(messages)\n",
    "\n",
    "        # 将文本和视觉信息转化为模型可以接受的格式\n",
    "        inputs = processor(\n",
    "            text=[text_template],\n",
    "            images=image_inputs,\n",
    "            videos=video_inputs,\n",
    "            padding=True,\n",
    "            return_tensors=\"pt\",\n",
    "        ).to(\"cuda\")  # 移动数据到GPU\n",
    "\n",
    "        # 推理：生成输出\n",
    "        with torch.no_grad():  # 禁用梯度计算以节省内存\n",
    "            generated_ids = model.generate(**inputs, max_new_tokens=128)\n",
    "\n",
    "        # 剪裁掉输入部分，只保留新生成的部分\n",
    "        generated_ids_trimmed = [out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)]\n",
    "\n",
    "        # 解码生成的token IDs为人类可读的文本\n",
    "        output_text = processor.batch_decode(\n",
    "            generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False\n",
    "        )\n",
    "        \n",
    "        response = output_text[0]\n",
    "        try:\n",
    "            # 尝试解析 JSON 响应\n",
    "            parsed_response = json.loads(response)\n",
    "            return parsed_response\n",
    "        except json.JSONDecodeError as ex:\n",
    "            print(f\"json 解析错误：{text}\")\n",
    "            return None\n",
    "        \n",
    "        # 清理CUDA缓存\n",
    "        del inputs, generated_ids, generated_ids_trimmed\n",
    "        torch.cuda.empty_cache()\n",
    "        \n",
    "        return output_text\n",
    "\n",
    "    except Exception as e:\n",
    "        print(f\"推理过程中发生错误: {e}\")\n",
    "        # 清理CUDA缓存（如果出现异常）\n",
    "        torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "0421f4b6-1836-4f35-94ff-9ba3704161e4",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-12-23T10:23:26.830322Z",
     "iopub.status.busy": "2024-12-23T10:23:26.829928Z",
     "iopub.status.idle": "2024-12-23T10:23:26.833559Z",
     "shell.execute_reply": "2024-12-23T10:23:26.832823Z",
     "shell.execute_reply.started": "2024-12-23T10:23:26.830297Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 初始化变量\n",
    "results = []\n",
    "num_errors = 0\n",
    "train_file_fullpath = '/mnt/workspace/datasets/doraemon-bot/train/train_fullpath.json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "72243758-98f8-439a-9d9e-2c509968f1b5",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-12-23T10:23:27.295302Z",
     "iopub.status.busy": "2024-12-23T10:23:27.294912Z",
     "iopub.status.idle": "2024-12-23T10:23:27.299730Z",
     "shell.execute_reply": "2024-12-23T10:23:27.299181Z",
     "shell.execute_reply.started": "2024-12-23T10:23:27.295276Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import json\n",
    "def load_json_from_file(file_path):\n",
    "    try:\n",
    "        with open(file_path, 'r', encoding='utf-8') as file:\n",
    "            content = file.read().strip()  # 移除任何首尾空白字符\n",
    "            if not content:  # 检查是否为空文件\n",
    "                print(\"文件为空\")\n",
    "                return None\n",
    "            data = json.loads(content)\n",
    "            return data\n",
    "    except FileNotFoundError:\n",
    "        print(f\"文件 {file_path} 未找到\")\n",
    "    except json.JSONDecodeError as e:\n",
    "        print(f\"JSON解码错误: {e}\")\n",
    "        print(\"请检查文件内容是否为有效的JSON格式\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "b720e839-47c1-4937-b099-61bae2db4c0e",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-12-23T10:25:50.102998Z",
     "iopub.status.busy": "2024-12-23T10:25:50.102627Z",
     "iopub.status.idle": "2024-12-23T10:25:55.079246Z",
     "shell.execute_reply": "2024-12-23T10:25:55.078595Z",
     "shell.execute_reply.started": "2024-12-23T10:25:50.102972Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "result:  {'output': '商品材质'}\n",
      "item[output]:  商品材质\n",
      "strtrsult:  商品材质\n",
      "results:  [True, True]\n"
     ]
    }
   ],
   "source": [
    "data = load_json_from_file(train_file_fullpath)\n",
    "if data is not None:\n",
    "    # print(\"成功加载数据:\", data)\n",
    "    for item in data:\n",
    "        text = item['instruction']\n",
    "        image_path = item['image'][0]\n",
    "        # print(image_path)\n",
    "        result = get_response(text=text, image_path=image_path)\n",
    "        print(\"result: \", result)\n",
    "        print(\"item[output]: \", item[\"output\"])\n",
    "        if result is not None and \"output\" in result:\n",
    "            strtrsult = result[\"output\"]\n",
    "            print(\"strtrsult: \",strtrsult)\n",
    "            results.append(strtrsult == item[\"output\"])\n",
    "        else:\n",
    "            num_errors += 1\n",
    "        break\n",
    "print(\"results: \", results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "00b9413a-c129-43c8-b20d-415bcccc469d",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2024-12-23T10:15:03.240771Z",
     "iopub.status.busy": "2024-12-23T10:15:03.240399Z",
     "iopub.status.idle": "2024-12-23T10:15:03.245226Z",
     "shell.execute_reply": "2024-12-23T10:15:03.244656Z",
     "shell.execute_reply.started": "2024-12-23T10:15:03.240747Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(26, 76, 0.34210526315789475)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(results), len(results), sum(results) / len(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "502af939-74d4-48a3-9f40-f2ba2687cdcc",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
